#scrap index items
from bs4 import BeautifulSoup
import re
from urllib.request import urlopen
from pymongo import MongoClient

index_url='https://www.tutorialspoint.com/index.htm'
client = MongoClient('mongodb://localhost:20000/')
db = client['ddbwork']

def getIndexItems(url):
    html = urlopen(url)
    bsObj = BeautifulSoup(html, 'html.parser')
    h4s = bsObj.find('div',{'class':'row featured-boxes'}).findAll('h4')
    for h4 in h4s:
        topclass = h4.get_text()
        print(topclass+'----------------------------------------')
        items = h4.nextSibling.nextSibling.findAll('li')##为什么要两个nextSibling?
##        i= 0
        for item in items:
            name = item.find('a').get_text()
            db[topclass].insert_one({'name':name})
            print(name)


getIndexItems(index_url)
